{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sklearn.metrics使用大全\n",
    "参考[官网资料](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics)，[相关算法](https://scikit-learn.org/stable/modules/model_evaluation.html#classification-metrics)。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入相关的包\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        bird       0.00      0.00      0.00         1\n",
      "         ant       0.67      1.00      0.80         2\n",
      "         cat       0.67      0.67      0.67         3\n",
      "\n",
      "    accuracy                           0.67         6\n",
      "   macro avg       0.44      0.56      0.49         6\n",
      "weighted avg       0.56      0.67      0.60         6\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 输出精确和召回\n",
    "from sklearn.metrics import classification_report\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")  # 忽略当某个类别某有预测到时出的警告\n",
    "\n",
    "y_true = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n",
    "y_pred = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n",
    "labels=list(set(y_true))\n",
    "print(classification_report(y_true, y_pred,labels=labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上表中的参数解释\n",
    "\n",
    "|参数|解释|\n",
    "| ------------ | ---- |\n",
    "| precision          | 精确率 |\n",
    "| recall         | 召回率 |\n",
    "| f1-score          | f1 值 |\n",
    "|    support          |    总数  |\n",
    "| accuracy     |   准确率   |\n",
    "| macro avg    | 宏平均 |\n",
    "| weighted avg | 加权平均 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.66666667 0.66666667]\n",
      "[0.         1.         0.66666667]\n",
      "0.6666666666666666\n",
      "[0.         0.8        0.66666667]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score,recall_score,accuracy_score,f1_score\n",
    "\n",
    "print(precision_score(y_true, y_pred, average=None, labels=labels))\n",
    "print(recall_score(y_true, y_pred, average=None, labels=labels))\n",
    "print(accuracy_score(y_true, y_pred))\n",
    "print(f1_score(y_true, y_pred, average=None, labels=labels))\n",
    "\n",
    "\n",
    "f1=f1_score(y_true, y_pred, average=None, labels=labels)\n",
    "f1[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1, 'Confusion matrix,without normalization')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAEbCAYAAAAVjwXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3debwcVZn/8c/3ZgUEggn7FsAIUYKySEQyEFmGiIqoKIugqBhxdAYRRv25BhUVhYAYgYmAEUWQAQcIoAwIAVkDyBAwIayBBAhJWAxrILnP749TN3Savt11b3qre7/v16te99Z26umq7qdPnzpVpYjAzMyKpaPVAZiZWc85eZuZFZCTt5lZATl5m5kVkJO3mVkBOXmbmRWQk3dG0iRJT0sKSUfWobyRWVm71CG8tidphqQpTdzeJEn31VimXx2DRpE0PtuPIyqNN3C7TX1PFU1bJ29JG0r6haSHJS2T9ISkP0vav87b2R74PnA0sDHwxzoUOz8r6//qUFbT9SLxfQz4f42MqczJwJ5dI5KmSbqiidtfRT9LNLeQ3tvP1KMwSUdKerHCrGa/pwplYKsD6I6kkcDNwAukA3gP6ctmb+AsYIs6bu5t2d9Lo05XLUXECmBhPcpqZ5IGR8RrEfFsM7cbES8ClT7w/Y6kDkDZe67hIuI1mvDebvZ7qnAioi0H4CrgSeAtFeatV/L/FsD/kJL8C8CfgM1K5k8C7gMOAR7OlrkUGFEyP0qHbPo04Iqy7U4C7isZHwP8FVialXsP8P5s3sisvF1Klt8DuB14FXgaOBUYXDJ/BnAG8GNgCbCIVMPsqLKfjiQlsQ8A9wMvA5cD6wIHAQ8C/wR+B6xRst4E4G/Ac8CzwNXA6JL5UTbMKN0vwDeABcCiktinZP9vC7wEfLpse68B7+3mdfwROLNk/MRsu2NLpi0APlV+LCodQ2B8yTH4OHBNtm9mA/uWbTvPcZlSts7K90f2f/n2R3bzOmseY2A94LfZsXkFuBZ4Z4Vjvj/pvb0c2L7s2CzMjvtPSZWeSdm2FgLfKIvpa8Cs7Jg9AZwNDCuZPz57TSO6GZ9X4fWv3AfVyi8pq3SYVGm/92C/7J3tl5eA64GtWp3PGjG0ZbOJpLeSPuxTItWwVhERz2XLiZSINwT2At4PbAJcms3rMhI4GPgo8K/AjqTkAOmD84Xs/42zIa8/AE8Bu2ZlTiIlgEqvaVPgz8Dd2bKfBw4FflK26KdIH8b3AV8BvprFXs0Q4Lhs3b2BXYCLgc+QEteBwIeAfytZZy3gtCz28aQP+nRJg7P5u2Z/J5D2ycdK1t0T2CGbt3d5MBExFzgWmCJpG0nrkxLLiRFxWzevYQbp+HUZT0pu7weQNArYNFuu3MnARaQPc9cxvKVk/onA6cC7gDuACyW9JSs373Gp5hjgVuA3JdufX2X5Wsd4GjAW+AjpOLwM/EXSGiXLDAW+A3wReAfwWDZ9D2Ar0v47Gvg6qSI0BBhHeo/+VNLOJWV1ZjG8Ezgs2+Yvc73y5D288bo3Jn2B3E/6IqxV/i3ZvJdL1j+5m+1Mo/Z+GUL6pf45YDdgGOmXet/T6m+Pbmonu5K+gT9aY7l9gRWU1HKArUlvln2y8UmkhLpuyTLfBh4qGT+IrMZdMm0atWveS4HPdBPbSEpq3qQE8hCr1rCOBJYBa8YbNY1by8q5Bji7yj44MtvOtiXTTs72y4hqr6esnLWydcZVir+snMXAkLLpM3hz7fR/gNuAK0lNYAOqbH90tr2NgTWz/fJN4Ops/heAB6sci0rHq+s1fLFk2qbZtK7Xmfe4dFvz7m6Zbl5n1WMMjMri26Nk/rqkL9ejyo75zhViml+6n4E7gVlly80Djq8S44Ts9Xdk4+OpUvMuW/cbpC/dbXpQ/pHAi93sq65fcz3ZL6WfhU+RfvF1++u1qENb1rwB1V4ESB/4JyNiXteEiHiE1NzyjpLlHouIf5aMPwlssLpBApOBsyVdJ+nbkrarEeutEdFZMu0mYDBvtLlD+nlZKk+syyLVdrs8DSyMiCVl01aWk9WI/5CdDF6aze8g37mE+yJiWY7ljiK9tj2Aw6NKm2xEzMliGA/sTmriuhDYXdKgbPqMHNuspHSfPpn97doXeY9LPVU7xqNJlY9bu2Zm7917WfU9vZzKJ8Nnl+3np7N1KZtW+l7YS9I1khZI6mp6HAxslPsVpXI+DJwAfDwiHq5z+Xn3S/ln4UlgEKkG3qe0a/J+kPQNOrrGcsqWq6R0+usV5tV67Z28+Utk0CqFREwivXEuJf0EniXpcy2IdXmFdWqVMx1Yn/SzeyypyWA56UNVy0s5loHUDrsu6Sf+pjmWv4HUTDIeuD77Ul5C+lm+J71P3iv3RWTVMd7YF3mOS833Qm/jKdlOaTzdKY1zWTdfhpXK7nZ7krYk/TKaA3wC2JnU5AD53gtk5WwPnA98JSJuKJlel/LJv18qfRagfXNdr7XlC4p0lvlq4CtdbZOlJHV9i84GNs16pnTN25rU7j17NcNYzJvbv99dIdYHI+L0iPggcA6ptlnJbGC3rGdAl3Gkn3QPV16lMSQNJ30x/jgirs1qvWuzau+j17K/A3q5jWHAeaQmnF8Bv5O0To3VZvBG8p6RTbsBmEj37d2l8fYm1jzHpdJ74V112n6leDpI7bUAZPttDKv/nq5kF1ISPTYibo2IB0ifn9yy/t6XA7+OiLN7UX6efdfs/dL22jJ5Z/6N9G17p6RPSNpW0naSvsQbPzuvJfXwOF/Szlmf5POBvwPXreb2rwN2lPQ5SW+T9HXSz3kAJK0h6VfZBQsjJY0lfei7eyOdQXrTniFptKQPknoCTImIl1cz1p56jlSj/UL22vYkndQprbUsIp3R3y/rb79uD7dxVraN75HaQV8gJfFqZpCaKnbljUQ9AzicdI7iiSrrzgO2z94nI7KmljzyHJfrgA9IOiArfzKweYXt75q9F0aUfRnkFhEPApcB/yXpXySNAX5POr/yh96UWcODpDzwVUlbSTqUdAKxJy4hNU+cImmjkmFAzvLnAUMl7ZvtuzXLN9CC/dL22jZ5R8SjwE6kkzknkRL2dcABpJ/6XT+BDyTVjGaQugUtBA4s+Xnc2+1fTWq/OxG4i3Ty64ySRVbwRteluaSTc7eSukVVKu8JUne+HUltlecCFwDfWp04eyNr3z2Y1GPkPlJS/S7pJFLXMsuB/yD9kniS9MHJRdIRpOP0qYh4PWsfPww4KPvwll6lN75km3NIx29uRCzOJl9PqpXNqLHZX5N+mt9Jej/sXn3xldvMc1zOLRluJnVH+5+yok4m1SBnZ9tfnesQPgvMJNVmZ5JO4E6IiFdWo8yKImIWqbfM10ixHwUc38Ni9iDt7ydIva+6hs3zlB8Rt5C+7C8g7buvd7Odpu2XItBq5jizXpH0WVINd9uIeL7V8ZgVTdvWvK3P2590sYgTt1kvuOZtZtZAkjYnnbzfiNRzaWpE/KJsGQG/IFVqXgaOjIi/Vyu3be9tYmbWRywHjouIv0taG7hL0jURUdq54QOkC5FGkbrunpn97ZabTczMGiginuqqRUfEC6QT6+XXPXwEOC+S24BhkqreqqMwNe8rB23r9p0G+8mEqa0Owawubpq+Z96rtLvVk5zzoeUPfJF0PUKXqRHxpg9Udk3KjqQboZXalFXvh7Mgm/ZUd9ssTPI2M2tXWaKuWvvJLji8BPhqRCwtn12p2GrlOXmbmVWgQatdeX+jrHTR2CXA+RHxpwqLLGDVC78244378FTk5G1mVsGANepxt4OVPUnOAeZExORuFrucdDuQC0knKv8ZEd02mYCTt5lZRR0D61bz3h04ArhXUtedIL9FdhVuRJxFuuf6/qTbE79Mupq0KidvM7MK6tVsEhE3UeM219ntPL7ck3KdvM3MKqhjzbshnLzNzCqo5wnLRnDyNjOrwDVvM7MCGjC4vS9Ad/I2M6tAHa55m5kVjga45m1mVjgdA1zzNjMrHDebmJkVkE9YmpkVkDqcvM3MCsfNJmZmBeQTlmZmBeSat5lZAbnN28ysgAYMcvI2MyscN5uYmRWQm03MzArINW8zswJy8jYzK6COgfV5enyjOHmbmVXgi3TMzArIzSZmZgXk3iZmZgXkmreZWQE5eZuZFZB7m/QzQzfbiHf/5mcM2XAE0dnJ4+dcxLxfntfqsPqksTutxzFfeBsdHeKKa57i9xfPb3VIfU5/3sdu8+5nYvkKZn/9pyy9ezYD3rIW426/hCXX3syLcx5udWh9SkcHfO3oURz73VksemYZZ0/eiZtuf4Z5819udWh9Rr/fx2rvZpP2/mopoGULF7P07tkArHjxJV68/xGGbrJhi6Pqe0aPWocFT73Ck0+/yvLlwbU3LmLc2OGtDqtP6e/7WB3KPbSCk3cDrbHlpqz77tE8P/OeVofS56w/fDCLlixbOb74mWWsP3xICyPqe/r7PlZHR+6hFZrSbCJpOhDdzY+IA5oRRzMNWGtNdr7odGYf92OWv/BSq8Ppcyr9oo1u32HWG/19H7u3SXJy9vdjwEbA77PxQ4F53a0kaSIwEeArHRswoWNYA0OsHw0cyM4Xnc4TF0xn4aXXtDqcPmnRktfYYMQbtcD1hw9hybPLqqxhPdXf93G79zZpSn0/Im6IiBuAHSPi4IiYng2HAeOqrDc1InaJiF2KkrgBdvj1ibx4/yM8etq0VofSZ93/4FI232QNNt5wKAMHin322ICbZz7T6rD6lP6+j9u9zbvZvU3Wl7R1RDwCIGkrYP0mx9BQ6+2+M5sdfiBL753LuDsvBWDudyaz+C83tjiyvmVFJ0w+6yEmnzCGjg5x5bULefTxftILokn6/T52V8FVHAvMkPRINj6SrFmkr3ju5ru4ctC2rQ6jX7jtrme57a5nWx1Gn9af97HavKtg05K3pA5gKTAK2C6bfH9E9J9GNDMrjHr2IpF0LvAhYFFEbN/NMuOB04BBwJKI2LNamU1L3hHRKemUiNgNcN85M2trqu8Jy2nAFKDi5daShgFnABMi4nFJG9QqsNmNOv8r6eNq998jZtbv1fOEZUTcCFRrfzoM+FNEPJ4tv6hWmc1u8/4asBawXNKrgICIiHWaHIeZWVWppTfvsm90a85MjYipPdjc24FBkmYAawO/iIiqN0VqavKOiLWbuT0zs17rQRfALFH3JFmXGwjsDOwNrAHcKum2iHig2goNJ2m7iLhf0k6V5kfE35sRh5lZXk2+7H0B6STlS8BLkm4E3gW0NnmTmksmAqew6mXyysb3alIcZma5NPnim8uAKZIGAoOBscCp1VZoSvKOiK62oP2BfyNdVRnA34AzmxGDmVlPaED9eptIugAYD4yQtAD4PqlLIBFxVkTMkfQXYBbQCZwdEfdVK7PZJyx/S+rrfXo2fiip68wnmxyHmVl1dWw2iYhDcyzzc+DnectsdvLeNiLeVTJ+vST3+TazttPuPZqb3c/7bknv7RqRNBa4uckxmJnV1tGRf2iBZvU2uZfUxj0I+LSkx7PxLYHZzYjBzKwnfD/v5ENN2o6ZWX304CKdVmhWb5PHmrEdM7N6qWdvk0bw0+PNzCpxs4mZWfG06sHCeTl5m5lV0uZdBZ28zcwqcc3bzKx4fMLSzKyI3FXQzKyA3NvEzKx4evIknVZw8jYzq8Q1bzOzAnLN28ysgNzbxMysgFzzNjMrILd5m5kVkGveZmYF5HubmJkVkO9tYmZWQB3ubVIXP5kwtdUhmK22807brNUhWF6ueZuZFZDbvM3MCsi9TczMCqioNW9J++ctJCKuqk84ZmbtIQp8efwVOcsIoL1fpZlZTxW42WSNpkVhZtZuipq8I2JZMwMxM2sn0eZt3rm/WiTtJeliSXdL2iybdqSkPRsXnplZi6gj/9ACubYq6RPAdGAxsB0wOJu1JvDNxoRmZtZCUv6hBfJ+ZXwbODoivgQsL5l+C7Bj3aMyM2uxGDAg99AKeft5vx24scL0pcCw+oVjZtYm2vyEZd7oFgJvqzB9d+CR+oVjZtYeQh25h1oknStpkaT7upn/KUmzsuEWSe+qVWbe5H0OcJqknUn9ujeUdDDwc8B3jDKzvqe+bd7TgAlV5j8K7BkROwA/JEdezdts8mPgraQ27kHAzaS2719ExGk5yzAzK4w8NercZUXcKGlklfm3lIzeBtS8/WSu5B0RARwn6QfAGFKN/d6IeC7P+mZmhdOD+3lLmghMLJk0NSJ62yrxeeDPtRbq6Y2pXiK1fwO80NOIzMyKoicX6WSJerWbkCW9n5S8x9VaNm8/70GSfgo8D8zNhuclnSRpcPW1zcwKqMkX6UjaATgb+EhEPFNr+bw17ynAAcAxwK3ZtN1IDevDgC/2PFQzs/YVNO/iG0lbAH8CjoiIB/Kskzd5HwIcHBF/KZk2W9KTwIU4eZtZH1PPE5aSLgDGAyMkLQC+T+r8QUScBXwPGA6codRcszwidqlWZt7k/SrwWIXp84DXcpZhZlYc9e1tcmiN+UcBR/WkzLzRnQl8q7R9W9Ig0n1NzuzJBs3MiqCzY0DuoRWqPUnnorJJE4B/lXR3Nv5u0j2/r25QbGZmrdPmt4St1myyomz8yrLx6+sci5lZ26hnm3cjVHsYQ9U2GjOzvqyZvU16w0+PNzOroLA173KSDgUOBbbgjYcxABAR76hzXGZmrdXmbd55r7D8KnAW8DDpSTrXAfOBTYCLGxadmVmLdGpA7qEV8v4u+BIwMSKOBV4HJkfEfsDpwPqNCs7MrFXqeT/vRsi71c1JtykEeAVYO/v/d8An6x2UmVmrBco9tELe5P006X7eAI8Du2b/bwltfkrWzKwX2r3mnfeE5fXAh4C7gd+SnqrzMWAscFmDYjMza5me3BK2FfIm76O7lo2IX0paSnp+5V+BXzYoNjOzlmnVici88j5J5zVKbkAVEb8l1cCtgrE7rccxX3gbHR3iimue4vcXz291SH2S93NjTT71VGbOnMmwYcM468z+dwujwl6kIyl33+2ImF2fcIqvowO+dvQojv3uLBY9s4yzJ+/ETbc/w7z5L7c6tD7F+7nx9t1nHw748Ic5+ZRTWh1KSxT5Ip37SE+Kr0TZvK6/uX5fSNoqIh6tNa3IRo9ahwVPvcKTT78KwLU3LmLc2OFOKnXm/dx4Y8aM4emnn251GC1T2Jo3MLoB27sE2Kls2sXAzg3YVkusP3wwi5YsWzm++JllvOPt67Qwor7J+9karbA174iYW6+NSNoOeCewbtZLpcs6wNAq6618IvM2Y45joy0/XK+QGqbSCero7veL9Zr3szVakWve9bQtqavhMKA0A78AfKG7lUqfyDzuwzcU4qO5aMlrbDBiyMrx9YcPYcmzy6qsYb3h/WyN1pn7MpjWaEryjojLgMsk7RYRt9ZcocDuf3Apm2+yBhtvOJTFzyxjnz024IST57Q6rD7H+9kaLZy8V/GQpG8BI0u3HRGfa3IcDbOiEyaf9RCTTxhDR4e48tqFPPq4T6LVm/dz4/30pJOYNWsWS5cu5fAjjuCIww9nv/32a3VYTdPuzSaKJjYUSroF+BtwFyVP6omIS2qtW5RmE7Nqzjtts1aH0C9svc02q5155z48P3fO2XabzZue6XtU85b0FmAbYHZEvN6L7a0ZEd/oxXpmZk3V7jXvvPfzXkvSecBSUq1582z6FEnf7sH2rpC0f8/DNDNrrr5yV8GfkB7C8D7g1ZLp/wt8ogfbO4aUwF+RtFTSC9l9UszM2kpndOQeWiFvs8lHgE9GxO2SStuBZgNb591YRKwt6a3AKKr07zYza7V2bzbJm7zXBxZVmL5WTzYm6ShS7Xsz4P+A9wK3AHv3pBwzs0Zr9+Sdt75/F1DaVt1V+/4c0JN+28cA7wEei4j3AzsCS3qwvplZU0Qo99AKeWve3wauyi5zHwh8WdI7gfHAnj3Y3qsR8aokJA2JiPslbduzkM3MGq+zL9S8I+JGUpLeAHgC+BjwErB7RMzswfYWSBoGXApcI+ky4MmehWxm1nh95YQlEXEXcPDqbCwiPpr9O0nS9cC6wF9Wp0wzs0Zo9zbvXMlb0prV5kdEj69LjogberqOmVmztKotO6+8Ne8X6f7BDJDzYQxmZkXRJ2rewAfKxgeReoocBXy3rhGZmbWBPlHzjoirK0y+QtIDwOHAeXWNysysxTpbHUANq3tL2DuBc+sRiJlZO2lVL5K8eh2dpMHAl0ldB83M+pR6XqQjaYKkuZIekvTNCvO3kHS9pLslzcpzA7+8vU0Ws+oJS5EeafYa8Ok8ZZiZFUm9TlhKGgD8CtgXWADcIenyiJhdsth3gIsi4kxJ7wCuIj20plt5m02+UzbeCSwGbomISvc8MTMrtM76Pf5lV+ChiHgEQNKFpJv9lSbvID2QHdL1LzUvXqyZvCUNBF4HroqIhT0M2syskHpS85Y0EZhYMmlq9gB1gE2B+SXzFgBjy4qYBPyvpH8n3fBvn1rbrJm8I2K5pCnA6FrLmpn1FT3pKpgl6qndzK5UUHm9/lBgWkScImk34HeSto+Ibju95G02mQm8C3gs5/JmZoW2on79vBeQPX0ssxlvbhb5PDABICJulTQUGEHlW3ED+ZP3FOAUSZuQbg/7UunMsoZ3M7PCq+NFOncAoyRtReqddwhwWNkyj5OeazBN0mjSw2oWVys0b/K+KPt7Rva3q8qv7H9fHm9mfUrU6YRl1vT8FeBqUq48NyL+IekHwJ0RcTlwHPBrSceScuqREdUjyJu83d5tZv1KPe9tEhFXkbr/lU77Xsn/s4Hde1Jm1eQt6VzgmIiY25NCzcyKro5dBRui1hWWnwHWaEYgZmbtpLNTuYdWqNVs0t631TIza5B2fwxanjbvNv/xYGZWf/U6YdkoeZL3Qqn6N1BEuLeJmfUpfeF+3hOB5xsdiJlZO2n3E5Z5kvd033zKzPqbojebtHn4ZmaNUcfL4xuiML1Nzjtts1aH0Od9+qsLWh1Cnzdnu5r32Lc62Pr11b80pdA174g2fw6QmVmDFDp5m5n1V50FbzYxM+uXXPM2MyugFd0+BqE9OHmbmVXQFy7SMTPrd9xsYmZWQH3hCkszs37HNW8zswJy8jYzKyD3NjEzK6BOJ28zs+Jxs4mZWQE5eZuZFZC7CpqZFVD0qOrd/KsxnbzNzCpYsaLVEVTn5G1mVoHbvM3MCsht3mZmBeSat5lZAUWPqt4+YWlm1hZ8ebyZWQF1tnmjt5O3mVkFbvM2MysgJ28zswLqbPPs3dHqAMzM2lF05h9qkTRB0lxJD0n6ZpXlDpIUknapVaZr3mZmFaxYUZ+at6QBwK+AfYEFwB2SLo+I2WXLrQ38B3B7nnJd8zYzqyAicg817Ao8FBGPRMRrwIXARyos90PgZ8CreeJz8jYzq6Az8g+SJkq6s2SYWFLUpsD8kvEF2bSVJO0IbB4RV+SNz80mZmYV9OQKy4iYCkztZnalyy9XFi6pAzgVOLIH4Tl5m5lVUsfOJguAzUvGNwOeLBlfG9gemCEJYCPgckkHRMSd3RXq5N0Ak089lZkzZzJs2DDOOvPMVofTZ43daT2O+cLb6OgQV1zzFL+/eH7tlSy3oZttxLt/8zOGbDiC6Ozk8XMuYt4vz2t1WE1Txyss7wBGSdoKeAI4BDisa2ZE/BMY0TUuaQZwfLXEDW7zboh999mHH/3wh60Oo0/r6ICvHT2K4yfdy+FfvoN99tiAkZuv2eqw+pRYvoLZX/8pN+ywPzePO5gtjz6Mt4zeptVhNU3nisg9VBMRy4GvAFcDc4CLIuIfkn4g6YDexueadwOMGTOGp59+utVh9GmjR63Dgqde4cmn04n5a29cxLixw5k3/+UWR9Z3LFu4mGULFwOw4sWXePH+Rxi6yYa8OOfhFkfWHPW8SCcirgKuKpv2vW6WHZ+nzKbWvCXtnmeaWS3rDx/MoiXLVo4vfmYZ6w8f0sKI+rY1ttyUdd89mudn3tPqUJqmjl0FG6LZzSa/zDnNrCpVOH/f5lczF9aAtdZk54tOZ/ZxP2b5Cy+1Opym6eyM3EMrNKXZRNJuwPuA9SV9rWTWOsCAKutNBCYC/OhHP+LQQw5paJxWHIuWvMYGI96oaa8/fAhLnl1WZQ3rDQ0cyM4Xnc4TF0xn4aXXtDqcpmr3ykCz2rwHA2/Jtrd2yfSlwEHdrVTad/KRhx9u811pzXT/g0vZfJM12HjDoSx+Zhn77LEBJ5w8p9Vh9Tk7/PpEXrz/ER49bVqrQ2m6FW3+NIamJO+IuAG4QdK0iHisGdtspZ+edBKzZs1i6dKlHH7EERxx+OHst99+rQ6rT1nRCZPPeojJJ4yho0Ncee1CHn3cJyvrab3dd2azww9k6b1zGXfnpQDM/c5kFv/lxhZH1hw9ewxa8zW7t8nLkn4OvBMY2jUxIvZqchwN9c1vfKPVIfQLt931LLfd9Wyrw+iznrv5Lq4ctG2rw2iZdk/ezT5heT5wP7AVcAIwj9SB3cysrfTk3iat0OzkPTwizgFej4gbIuJzwHubHIOZWU3RGbmHVmh2s8nr2d+nJH2QdH3/Zk2Owcysplb1386r2cn7R5LWBY4j9e9eB/hqk2MwM6up3XubNLvZ5BOAIuK+iHg/6ckSH21yDGZmNbnZZFU7RMTzXSMR8Wx2E3Izs7bS7r1Nmp28OyStFxHPAUh6awtiMDOrqd2fHt/sxHkKcIuki0lPkvgkcGKTYzAzq8k17xIRcZ6kO4G9SI8G+lj5E5TNzNqBe5uUyZK1E7aZtbUVy9u7t4nbm83MKnDN28ysgKLTNW8zs8Jp1UMW8nLyNjOrwM0mZmYF1OkTlmZmxdMZTt5mZoXji3TMzArIydvMrIB8wtLMrIA63c/bzKx4OlesaHUIVTl5m5lV4DZvM7MCcvI2Mysg9/M2Mysg17zNzArIdxU0Mysg9zYxMysg3xLWzKyA2r3ZpKPVAZiZtaPojNxDLZImSJor6SFJ36wwf4ikP2bzb5c0slaZTt5mZhVEdOYeqpE0APgV8AHgHcChkt5Rttjngeci4m3AqcBJteJz8jYzq6Bz+YrcQw27Ag9FxCMR8RpwIfCRsmU+Avw2+/9iYG9JqlNU+1EAAAdzSURBVFZoYdq8t95mm6ovpB1JmhgRU1sdR143Td+m1SH0WNH2McxtdQA9Vrx9XB83Td8zd86RNBGYWDJpask+2xSYXzJvATC2rIiVy0TEckn/BIYDS7rbpmvejTWx9iK2mryPG8/7uIaImBoRu5QMpV92lb4EyhvK8yyzCidvM7PGWgBsXjK+GfBkd8tIGgisCzxbrVAnbzOzxroDGCVpK0mDgUOAy8uWuRz4TPb/QcB1UeNpEIVp8y6oftdO2ALex43nfbwasjbsrwBXAwOAcyPiH5J+ANwZEZcD5wC/k/QQqcZ9SK1y1e6P+jEzszdzs4mZWQE5eZuZFZCTdy9JGinpvgrTz65w9VSl9cdLuqIx0fVvkg7Mcwysd7L37vtaHUd/5+RdZxFxVETMLp+eXSJrzXEg6TJka4zxgJN3izl5r56Bkn4raZakiyWtKWmGpF0AJL0o6QeSbgd2y25Oc7+km4CPtTb0YpF0qaS7JP0ju5qta/+eKOkeSbdJ2jCrER4A/FzS/0kq3mWjLSLp09l7+R5Jv5P04ewmSXdLujbbvyOBo4Fjs/37L62Nuv9yb5Neyt7EjwLjIuJmSecCs4EPAcdHxJ2SAjg4Ii6SNBR4ENgLeAj4I7BmRHyoJS+gYCS9NSKelbQGqd/snqRLhw+IiOmSfgYsjYgfSZoGXBERF7cw5EKR9E7gT8DuEbFE0ltJV/g9HxEh6ShgdEQcJ2kS8GJEnNzCkPs917xXz/yIuDn7//fAuLL5K4BLsv+3Ax6NiAezzve/b1KMfcV/SLoHuI10Jdoo4DWg67zBXcDI1oTWJ+wFXBwRSwAi4lnSlYBXS7oX+E/gnS2Mz8o4ea+e8p8t5eOvRsSKKvMtB0njgX2A3SLiXcDdwFDg9ZKr0Fbgi85Wh3jz+/OXwJSIGAN8kbTPrU04ea+eLSTtlv1/KHBTlWXvB7YqaYM9tKGR9S3rku51/LKk7YD31lj+BWDtxofVp/wV+KSk4ZCaqUj7/Yls/mdKlvX+bQNO3qtnDvAZSbOAtwJndrdgRLxKujvbldkJy8eaE2Kf8BfSyeFZwA9JTSfVXAj8Z3aizScsc4iIfwAnAjdkzVOTgUnAf0v6G6vemnQ68FGfsGwtn7A0Mysg17zNzArIydvMrICcvM3MCsjJ28ysgJy8zcwKyMnbWk7Sfdkl113j8yQdX+dtjJcUkkb0YJ0Zkqas5nZHZtvdZXXKMSvn5G1vImlalnBC0uuSHpF0sqS1mhTCe4Az8iwo6UhJLzY4HrO248uJrTvXAkcAg4B/Ac4G1gK+VGlhSYMi4vV6bDgiFtejHLO+zDVv686yiFgYEfMj4g/A+aT7ZJc2Qewvaaak14D9snkfzm7d+qqkR7Nbtg7uKlTSBpIuk/SKpMckfa58w+XNJpLWkXSmpKeycudIOji758lvgLVKfilMyvPiJA2XdIGkBVks/5D02QqLDpT0C0nPZcPPJXWUlDNY0klZOS9JukPSfnliMFsdrnlbXq+QauGlTgKOI93i9oUsaZ0PHAPcCGwBnAUMAbqS8TRgS9KNpl4GTqXK3QAlCfgzsB7wWeABYFvSTZJuAb4K/Bjougw+bxPKUODv2WtYmsXzX5Iej4i/liz3qSzm3YAdgF8DT5EuH4f05bENcBiwANgfmC7pPRFxT85YzHouIjx4WGUgJasrSsZ3Jd3b4o/Z+HjSHeg+XrbejcB3y6YdSEqoAt6erbd7yfwtSXcEnFQybR7pnugA+wKdpHtJV4r1SNK9pWu9pq6YR1RZ5kLg7JLxGaQvC5VM+w6wIPt/myy2LcrKuRQ4I/t/ZLbdXVp9XD30rcE1b+vOhOxE4EBSjfsy4N/LlrmzbHxnYFdJ3yiZ1gGsAWwEjCYlu5ldMyPiMUlPVoljR+CpiJjTq1fRDaXH0n0TOBjYlPTrYDApYZe6LSJKbwB0K/BDSesAO5G+lGanHwgrDQGuq2e8ZuWcvK07N5Lugvg68GRUPhn5Utl4B3AC8N8Vll1MSnQ91Zt18jie1ORzDHAv6dfBj4ENelBGB6lW/R7Sfir1Sh1iNOuWk7d15+WIeKiH6/wd2K679STNISW895Daq5G0BbBJjTI3ljS6m9r3a0BvHu48DpgeEb/L4uhq1nm+bLmxklRS+34v6ctsqaS7SV8uG0XE9b2IwazX3NvE6ukHwGFKD13eXtJ2kg7Kni9JRMwl3Zv7vyTtJundpPb1arXUvwK3A5dI2k/SVpL2lXRgNn8eMDSbNkLSmjljfQDYW9K47AEPU4CtKiy3CXCapG0lHUR6HNip2et5gHSCdlr2OreWtIuk4yX5AdPWUE7eVjcRcTXwQeD9pHbtmaR25cdLFjuS9ODm60g39f8DKQF3V2Yn8AHgZtJzP+cAvyC1TxMRt5B6tFxAapr5es5wf5TF92dSE9FLpERc7nxSzf52Uk+Tc8iSd+azpB4nPyM9LekKYA/8sA1rMD+MwcysgFzzNjMrICdvM7MCcvI2MysgJ28zswJy8jYzKyAnbzOzAnLyNjMrICdvM7MC+v8aTLH3i5S3xwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 绘制混淆矩阵\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "\n",
    "cm=confusion_matrix(y_true, y_pred)\n",
    "df = pd.DataFrame(cm, columns=labels,index=labels)\n",
    "sns.heatmap(df,annot=True,cmap=\"coolwarm\")\n",
    "plt.xlabel('Predict label', color='k', size=14) \n",
    "plt.ylabel('True label', color='k', size=14) \n",
    "plt.title('Confusion matrix,without normalization', color='k', size=14) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其中参数 `cmap` 颜色模式如下\n",
    "![](https://tva1.sinaimg.cn/large/007S8ZIlly1gfpignsgybj30l20jfk0w.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "准确率 0.6666666666666666\n"
     ]
    }
   ],
   "source": [
    "# 输出准确率\n",
    "from sklearn.metrics import accuracy_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")  # 忽略当某个类别某有预测到时出的警告\n",
    "\n",
    "y_true = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n",
    "y_pred = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n",
    "labels=list(set(y_true))\n",
    "\n",
    "print(\"准确率\", accuracy_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "y_true = np.array([0, 0, 1, 1])\n",
    "y_scores = np.array([0.1, 0.4, 0.35, 0.8])\n",
    "roc_auc_score(y_true, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de5xN9frA8c8zM+bGuAxSkVxzv9UkUohcQiXVQaI6nVNSCSU5VNJdInLt6nT6lUopp1yilHKIUSKXIsRI7sa4zJjL8/tjLWMbM3v2MHv2zJ7n/XrNy173Zy1772ev73d9v19RVYwxxpichAQ6AGOMMYWbJQpjjDFeWaIwxhjjlSUKY4wxXlmiMMYY45UlCmOMMV5ZoggCItJHRL4MdByBJiJVReSIiIQW4DGriYiKSFhBHdOfRGSdiLQ9i+2C9j0oIm1FJCHQcQSSJYp8JiLbROS4+4X1l4jMEJFS/jymqv6fqnb05zEKI/daX3tyWlW3q2opVU0PZFyB4iasWueyD1VtoKrf5HKcM5JjcX0PFheWKPzjelUtBTQFmgHDAxzPWQnkr+Rg+YWeF3a9TWFlicKPVPUvYAFOwgBARCJEZKyIbBeR3SIyTUSiPJbfKCKrReSwiPwuIp3d+WVE5E0R2SUiO0XkmZNFLCJyp4h8776eJiJjPeMQkc9EZIj7+kIR+VhE9orIVhEZ6LHeKBGZJSLvishh4M6s5+TG8Y67/R8iMlJEQjziWCoir4pIoohsFJH2Wbb1dg5LRWS8iBwARolITRH5WkT2i8g+Efk/ESnrrv8foCrwX/fu7dGsv3RF5BsRedrdb5KIfCkiFTzi6eeew34ReTzrHUqW844SkZfd9RNF5HvP/zegj/t/uk9ERnhs11xElonIIfe8J4lIuMdyFZH7RWQTsMmdN0FEdrjvgVUicrXH+qEi8i/3vZHkLr9IRJa4q/zsXo+e7vrd3PfTIRH5n4g09tjXNhEZJiJrgKMiEuZ5DdzY4904dovIOHfTk8c65B6rped70N22gYgsFJED7rb/yuG65vh5cGNb7vH/eZ84RWOR7vRH4ty1J4rIEhFp4LHfGSIyRUTmuTEuFZHzReQVETnovjebZbkWw0Vkvbv87ZPHySbmHD9DQUtV7S8f/4BtwLXu6yrAWmCCx/JXgDlALBAD/Bd43l3WHEgEOuAk8cpAXXfZp8B0oCRwHrACuNdddifwvfu6NbADEHe6HHAcuNDd5yrgCSAcqAFsATq5644CUoHu7rpR2ZzfO8BnbuzVgN+Auz3iSAMGAyWAnu75xPp4DmnAg0AYEAXUcq9FBFAR5wvqleyutTtdDVAgzJ3+BvgduMTd3zfAC+6y+sAR4Cr3Wox1z/3aHP5fJ7vbVwZCgSvduE4e83X3GE2AFKCeu91lQAv3nKoBG4BBHvtVYCHO+yHKnXc7UN7d5mHgLyDSXTYU5z1VBxD3eOU99lXLY9+XAnuAK9yY73CvWYTH9VsNXORx7MxrCiwD+rqvSwEtsrvO2bwHY4BdbuyR7vQVOVxXb5+HEPf/fBRQGzgINPPY9u/uNhHuflZ7LJsB7HOvfyTwNbAV6Odei2eAxVneS7+41yIWWAo84y5rCyR4xJTjZyhY/wIeQLD9uW+4I0CS+2H6CijrLhPgKFDTY/2WwFb39XRgfDb7rITz5RPlMa/3yTd6lg+pANuB1u70P4Gv3ddXANuz7Hs48Lb7ehSwxMu5hbpx1PeYdy/wjUccf+ImKXfeCqCvj+ewPadju+t0B37Kcq1zSxQjPZYPAOa7r58A3vdYFg2cIJtE4X45HAeaZLPs5DGrZDnnXjmcwyBgtse0Au1yOe+DJ48N/ArcmMN6WRPFVODpLOv8CrTxuH5/z+b9ezJRLAGeAirkcM45JYrenv9PXs7L6+fB41gHcBLscC/7KuvGVMadngG87rH8QWCDx3Qj4FCW8+7vMd0F+N193ZZTicLrZyhY/6xc0j+6q+oiEWkDvAdUAA7h/CqOBlaJyMl1BecLGJxfM3Oz2d/FOL/Qd3lsF4Jz53AaVVURmYnzYV0C3Aa867GfC0XkkMcmocB3HtNn7NNDBZxfUX94zPsD51f2STvV/fR4LL/Qx3M47dgich4wEbga55djCM6XZl785fH6GM4vY9yYMo+nqsdEZH8O+6iA86v097weR0QuAcYBcTj/92E4v0g9ZT3vh4F/uDEqUNqNAZz3iLc4PF0M3CEiD3rMC3f3m+2xs7gbGA1sFJGtwFOq+rkPx/U1xtw+D6jqNhFZjPPFPTlzJafI8lngVnc/Ge6iCjh3sQC7PY51PJvprA+ZeF6Lk+/brHz5DAUdq6PwI1X9FueXzck6g304b9AGqlrW/SujTsU3OG/UmtnsagfOr/EKHtuVVtUG2awL8D5wi4hcjPML6GOP/Wz12EdZVY1R1S6eYXs5pX04xTMXe8yrCuz0mK4sHp96d/mfPp5D1mM/785rrKqlcYpkxMv6ebELp2gQcOogcIp7srMPSCb7/5vcTAU2ArXdc/gXp58DeJyHWx8xDPgbUE5Vy+J88Z3cJqf3SHZ2AM9m+f+OVtX3szt2Vqq6SVV74xQTvgjMEpGS3rbJY4y5fR4QkS44dxlfAS95bHsbcCNwLVAG584Dzry2eXGRx+uT79usfPkMBR1LFP73CtBBRJqqagZOWfZ499cyIlJZRDq5674J3CUi7UUkxF1WV1V3AV8CL4tIaXdZTfeO5Qyq+hOwF3gDWKCqJ3/9rAAOu5WEUW7FaEMRudyXE1HnsdMPgWdFJMZNREM4dccCzpfKQBEpISK3AvWAuXk9B1cMTjHeIRGpjFM+72k3Thnx2ZgFXC8iV4pTufwUOXzJuP9vbwHj3IrMULcCN8KH48QAh4EjIlIXuM+H9dNw/v/CROQJnDuKk94AnhaR2uJoLCInE1zW6/E60F9ErnDXLSkiXUUkxoe4EZHbRaSie/4n30PpbmwZ5HztPwfOF5FBbmV1jIhckXWl3D4P4jx48CbO3dUdOP9fJ7+QY3B+eOzHuSt5zpdzysX9IlJFRGJxEvoH2axzTp+hosoShZ+p6l6cCuDH3VnDgM3AcnGeLFqEUzGJqq4A7gLG4/yK/JZTv9774RQbrMcpfpkFXODl0O/j/Np6zyOWdOB6nKewtuL8onsD5xeZrx7EKVfeAnzv7v8tj+U/4FQ87sMpGrhFVU8W6eT1HJ7CqZBNBL4APsmy/HlgpDhP9DySh3NAVde55zIT5+4iCafiNyWHTR7BqUReiVNm/iK+fX4ewfn1m4TzpZjdl4+nBcA8nIcE/sC5k/EsEhmHk6y/xElAb+JUooNTx/Rv93r8TVXjceqoJuFc781k8ySbF52BdSJyBJiAU++SrKrHcP5vl7rHauG5kaom4TyEcD1Okdwm4JocjpHj5wF4DfhMVee676G7gTfcxPiOe3124ryflufhvHLyHs513eL+PZN1hXz6DBU5J5+MMeacicidwD9U9apAx5JX4jSKPIRTRLQ10PGYgiUi23Deu4sCHUthZHcUptgSketFJNotdx+Lc8ewLbBRGVP4WKIwxdmNOBWWf+IUl/VSu8U25gxW9GSMMcYru6MwxhjjVZFrcFehQgWtVq1aoMMwxpgiZdWqVftUteLZbFvkEkW1atWIj48PdBjGGFOkiMgfua+VPSt6MsYY45UlCmOMMV5ZojDGGOOVJQpjjDFeWaIwxhjjlSUKY4wxXvktUYjIWyKyR0R+yWG5iMhEEdksImtE5FJ/xWKMMebs+fOOYgZON8U5uQ6nf53awD04A7wYY4zJZydOpJ/T9n5rcKeqS0SkmpdVbgTecTthWy4iZUXkAneAG2PMSZ90ha3ZjZBrTO6G/rcDP/3pbdiX3AWyjqIypw/IksDpYy9nEpF7RCReROL37t1bIMEZU2hYkjDnoOH5e/huS9Vz2kcgu/DIbtjJbLuyVdXXcEa7Ii4uzrq7NcXTw/bWN7lbv34vP/64i9tvbwxAP1XavJBI9epnDNjns0AmigROH8y8CtkPZm6MMSYXx46l8swzS3jppf8RGiq0aFGFWrViERGqVSt7TvsOZKKYAzwgIjOBK4BEq58wxpi8mzdvE/ffP5etWw8BcPfdl1G+fFQuW/nOb4lCRN4H2gIVRCQBeBIoAaCq04C5QBecgdWPAXf5KxZjjAlGO3ceZtCgBcyatR6Axo0rMW1aV1q2vCiXLfPGn0899c5luQL3++v4xhgT7O6/fy6fffYr0dElGD26LQ891IKwsPx/RqnIjUdhjDHFWVpaRmYyePHFaylRIpSXX+5I1apl/HZMSxTG+IO1fTD5LDExmZEjv+a33w4wf34fRIQ6dSrw0Ue3+v3YliiM8Yf8ThLVu+Tv/kyRoap89NF6Bg2az65dRwgNFVav/otmzc6tEV1eWKIwxp+s7YM5B7//foAHHpjH/PmbAWjZsgrTpnWjceNKBRqHJQpjjCmExo79H48/vpjk5DTKlo3kxRev5R//uJSQkOzaKvuXJQpjjCmEjh1LJTk5jb59GzN2bEfOO69kwGKxRGGMMYXA3r1H+fXX/Vx1ldMv07BhrWjbthqtW18c4Mhs4CJjjAmojAzljTd+pE6dSfTo8QEHDhwHICIirFAkCbA7CmOMCZhfftlD//6fs3Sp05F2hw41OHYsldjY/Ot+Iz9YojDGmAJ29OgJRo/+lnHjlpOWlkGlSiV55ZXO9OzZAJGCr6zOjSUKY3JjjedMPrvllo+YP38zIjBgQBzPPtuesmUjAx1WjixRGJObs00S1kjO5GDYsFbs3n2EqVO7csUVVQIdTq4sURjjK2s8Z85CWloGr776A9u2HWLChOsAaNu2GvHx9wSkTcTZsERhjDF+smLFTu6993NWr/4LgHvuuYwGDc4DKDJJAuzxWGOMyXeHDiUzYMAXtGjxBqtX/8XFF5fhv//tnZkkihq7ozDGmHw0c+YvDBo0n927jxIWFsLDD7fk8cdbU7JkeKBDO2uWKIwxJh99+eXv7N59lFatLmLq1K40alSwHfj5gyUKY4w5BykpaezcmUSNGuUAGDOmA1dfXZU77mhapOohvLE6CmOMOUtff72Vxo2n0bXre5w4kQ5AhQrR3HVXs6BJEmB3FMY4rFGdyYPdu4/wyCMLeffdNQDUrVuBhITDmXcVwcYShTGQe5KwxnMGpwO/119fxWOPfcWhQ8lERoYxcuTVDB3aivDw0ECH5zeWKIzxZI3qjBc33fQBc+b8CkCnTjWZPLkLNWvGBjgq/7M6CmOM8VGPHnU5//xSfPDBLcyb16dYJAmwOwpjjMnRnDm/kpBwmAEDLgegX78m9OhRj5iYiABHVrAsURhjTBbbtycycOA8PvvsVyIiQuncuRY1apRDRIpdkgBLFMYYkyk1NZ2JE3/gySe/4ejRVGJiwnnmmXZcfHGZQIcWUJYojDEGWL48gXvv/Zw1a3YDcOut9Rk/vhOVK5cOcGSBZ4nCBB9rE2HOwuOPL2bNmt1Ur16WSZO60KVL7UCHVGhYojDBxwYaMj5QVZKSTlC6tFPnMGnSdbzzzs+MGNGa6OgSAY6ucLFEYYKXtYkwOfj1130MGDAXEVi4sC8iQp06FXj22faBDq1QskRhjCk2kpPTeP7573jhhaWcOJFO+fJRbNt2iOrVg7PrjfxiicIYUywsXPg7AwbMZfPmAwD8/e9NGTOmA+XLRwc4ssLPry2zRaSziPwqIptF5LFsllcVkcUi8pOIrBERKyQ2xuQrVeXvf/+Mjh3fZfPmA9SvX5ElS+7kzTdvtCThI7/dUYhIKDAZ6AAkACtFZI6qrvdYbSTwoapOFZH6wFygmr9iMsYUPyJCtWpliYoK44kn2jBkSMug7sDPH/xZ9NQc2KyqWwBEZCZwI+CZKBQ4+ZByGeBPP8ZjjCkmVq/+i127krjuOucR12HDWtG3b2OrizhL/ix6qgzs8JhOcOd5GgXcLiIJOHcTD2a3IxG5R0TiRSR+7969/ojVGBMEkpJSGDJkAZdd9hp33PEpBw4cByAiIsySxDnw5x1FdsM7ZX1esTcwQ1VfFpGWwH9EpKGqZpy2keprwGsAcXFx9sxjsLEGcuYcqSqffrqRgQPnk5BwmJAQ4bbbGlGihHWQnR/8mSgSgIs8pqtwZtHS3UBnAFVdJiKRQAVgjx/jMoWNP5KENZ4rNv744xAPPDCPzz//DYC4uAuZPr0bl156QYAjCx7+TBQrgdoiUh3YCfQCbsuyznagPTBDROoBkYCVLRVX1kDO5JGqcvPNH7Jq1S5Kl47guefa0b9/HKGhdieRn/yWKFQ1TUQeABYAocBbqrpOREYD8ao6B3gYeF1EBuMUS92pqvZtYYzxKiNDCQkRRISxYzsybVo848d34oILYgIdWlCSova9HBcXp/Hx8YEOw+Snl93qLLujMLnYv/8Yjz22CIDXX78hwNEULSKySlXjzmZbuz8zxhR6qsq//72aunUn88YbP/HOO2tISDgc6LCKDevCwxhTqG3YsJf77vuCb7/9A4C2basxdWpXqlSxcSIKiiUKY0yhpKo88cRiXnxxKampGVSoEM3LL3ekb9/GiGT39L3xF0sUpuBYewmTByLCzp1JpKZm8M9/XsoLL1xLbGxUoMMqlixRmILjLUlYuwcD/PlnEvv2HaNx40oAjBnTgbvvbkarVlUDHFnxZonCFDx7uslkkZ6ewdSp8YwY8TWVK8ewenV/wsNDqVAhmgoVLEkEmiUKY0xA/fjjLu6993Pi452OG1q3vpjDh1OoUMG6AC8sfEoUIhIOVFXVzX6OxxhTTBw+nMLjj3/NpEkrychQqlQpzcSJneneva5VVhcyuSYKEekKjAPCgeoi0hR4UlVv8ndwxpjgpKq0bv02P/+8m9BQYciQFowa1ZaYmIhAh2ay4UuDu9HAFcAhAFVdDdTyZ1DGmOAmIgwe3ILmzSsTH38PL7/cyZJEIeZL0VOqqh7KcitotZHGGJ+dOJHOuHHLCA0Vhg5tBUC/fk24/fbG1oFfEeBLotggIn8DQtyeYB8Clvs3LGNMsPjuuz/o3/8L1q/fS0REKP36NaFSpVKICKGhVhdRFPiSKB4AngAygE9weoMd7s+gTBFmjeqMa9++Yzz66ELefns1ALVrxzJlSlcqVSoV4MhMXvmSKDqp6jBg2MkZItIDJ2kYc7rckoQ1rAt6qsqMGasZOnQh+/cfJzw8lOHDr+Kxx64iMtKeyC+KfPlfG8mZSWFENvOMOcUa1RVr7767lv37j9OuXXWmTOlCnToVAh2SOQc5JgoR6YQzTGllERnnsag0TjGUMcYAcOxYKomJyVxwQQwiwpQpXVi58k/69GlkbSKCgLc7ij3AL0AysM5jfhLwmD+DMsYUHfPmbeL+++dSo0Y5Fi7si4hQp04Fu4sIIjkmClX9CfhJRP5PVZMLMCZjTBGwc+dhBg1awKxZ6wGIiYlg//7j1vVGEPKljqKyiDwL1AciT85U1Uv8FpUxptBKT89g8uSVjBz5NUlJJyhZsgSjR1/DwIFXEBZmbSKCkS+JYgbwDDAWuA64C6ujMKZYyshQ2rSZwdKlOwDo3r0uEyZ0pmrVMgGOzPiTL+k/WlUXAKjq76o6ErjGv2EZYwqjkBChY8eaXHRRaT77rBezZ/e0JFEM+HJHkSLOYwu/i0h/YCdwnn/DMsYUBqrKhx+uIywshJtvrg/AsGGtGDKkJaVKhQc4OlNQfEkUg4FSwEDgWaAM8Hd/BmWMCbzffz/AgAFz+fLL36lYMZp27apTrlwUERFhRFj/fcVKrolCVX9wXyYBfQFEpIo/gzLGBE5KShovvfQ/nn32O5KT0yhXLpJnn21HmTKRuW9sgpLXRCEilwOVge9VdZ+INMDpyqMdYMnCmCDzzTfbuO++L9i4cR8Affs2ZuzYjpx3XskAR2YCKcfKbBF5Hvg/oA8wX0RGAIuBnwF7NNaYIJOensGAAU6SqFOnPF9/3Y933rnJkoTxekdxI9BEVY+LSCzwpzv9a8GEZozxt4wMJTk5jejoEoSGhjB1aleWLPmDRx9tRUSEdeBnHN7eCcmqehxAVQ+IyEZLEsYEj7Vrd9O//xfUrVueN9+8EYA2barRpk21wAZmCh1viaKGiJzsIVaAah7TqGoPv0ZmjPGLo0dPMHr0t4wbt5y0tAy2bj3IwYPHKVcuKtChmULKW6K4Ocv0JH8GYozxv//+91ceeGAe27cnIgIDBsTx7LPtKVvWnmgyOfPWKeBXBRmIMcZ/0tIy6NlzFp98sgGApk3PZ/r0bjRvXjnAkZmiwGqrjCkGwsJCKFMmglKlwnn66Wt44IHm1oGf8Zlf3yki0llEfhWRzSKS7RgWIvI3EVkvIutE5D1/xmNMcfLDDwn88ENC5vRLL3Vgw4b7GTSohSUJkyc+31GISISqpuRh/VBgMtABSABWisgcVV3vsU5tYDjQSlUPioj1IWXMOTp0KJnhwxcxffoq6tatwOrV/QkPD6V8eRsnwpydXH9WiEhzEVkLbHKnm4jIqz7suzmwWVW3qOoJYCZO2wxP/wQmq+pBAFXdk6fojTGZVJX33ltL3bqTmDZtFaGhIdxwQx3S021UAHNufLmjmAh0Az4FUNWfRcSXbsYrAzs8phOAK7KscwmAiCwFQoFRqjrfh30bYzxs2rSfAQPmsmjRFgBatbqIadO60bCh3aSbc+dLoghR1T+yDJCe7sN22Y2ortkcvzbQFqfvqO9EpKGqHjptRyL3APcAVK1a1YdDG1N8pKam067dOyQkHCY2NooxY67lrruaERKS3UfQmLzzJVHsEJHmgLr1Dg8Cv/mwXQJwkcd0FZxuQLKus1xVU4GtIvIrTuJY6bmSqr4GvAYQFxeXNdkYUyypKiJCiRKhPPtsOxYv3saYMddSsaL1zWTyly+J4j6c4qeqwG5gkTsvNyuB2iJSHWewo17AbVnW+RToDcwQkQo4RVFbfAs9yH3SFbbODXQUphDavfsIjzyykEsuieXxx9sA0K9fE/r1axLgyEyw8iVRpKlqr7zuWFXTROQBYAFO/cNbqrpOREYD8ao6x13WUUTW4xRnDVXV/Xk9VlAqykmiepdARxCUMjKU119fxWOPfcWhQ8mULRvJoEEtiImxUYSMf/mSKFa6RUIfAJ+oapKvO1fVucDcLPOe8HitwBD3z2TnYStpM/Dzz3/Rv/8XLF/utIvo3LkWkyd3sSRhCoQvI9zVFJErcYqOnhKR1cBMVZ3p9+iMKeZSU9MZPvwrXnllOenpygUXlGLChM7cckt9sjxgYozf+NQ8U1X/p6oDgUuBwzgDGhlj/CwsLISffvqLjAzlwQebs2HD/dx6awNLEqZA5XpHISKlcBrK9QLqAZ8BV/o5LmOKre3bE0lPz6B69XKICNOmdSUxMYW4uAsDHZoppnypo/gF+C8wRlW/83M8xhRbqanpTJjwA08++Q0tW1Zh4cK+iAi1a5cPdGimmPMlUdRQVesDwBg/WrZsB/37f8GaNbsBiI2N4tixVEqWDA9wZMZ4SRQi8rKqPgx8LCJnPHpjI9wZc+4OHjzOY48t4rXXfgSgevWyTJ7cheuuqx3gyIw5xdsdxQfuvzaynT9Zw7piKyUljaZNp7N9eyIlSoQwdOiVjBjRmujoEoEOzZjTeBvhboX7sp6qnpYs3IZ0NgJefvCWJKzhWlCLiAjj7rub8dVXW5k6tSv161cMdEjGZEucNm9eVhD5UVUvzTLvJ1Vt5tfIchAXF6fx8fGBOLR/vOw+5mgN64JecnIazz//HXXqVOC22xoBzhCloaFij7savxORVaoadzbbequj6InzSGx1EfnEY1EMcCj7rYwx2Vm48HcGDJjL5s0HOO+8ktx0U12iokrYSHOmSPBWR7EC2I/T6+tkj/lJwE/+DMqYYPHXX0cYMmQB77//CwANGlRk2rRuREVZPYQpOrzVUWwFtuL0FmuMyYP09AymT1/Fv/71FYmJKURFhfHkk20YPLgl4eGhgQ7PmDzxVvT0raq2EZGDnD7gkOD05xfr9+iMKaLS05VXX11BYmIKXbrUZtKk66hevVygwzLmrHgrejo53GmFggjEmKIuKSmF9HSlbNlIwsNDef3169m9+wg9etSzympTpHkrejrZGvsi4E9VPSEiVwGNgXdxOgc0vrC2EkFNVZk9eyMDB86jU6eavPnmjQBcdZUN22uCgy+PXHyKMwxqTeAdnI4B3/NrVMEmtyRh7SWKrG3bDnHDDTO5+eYP2bkziV9+2UtyclqgwzImX/nS11OGqqaKSA/gFVWdKCL21NPZsLYSQSM1NZ1x45bx1FPfcvx4GqVLR/Dcc+3o3z+O0FB75NUEF5+GQhWRW4G+QHd3nj3bZ4qtY8dSadHiDdau3QNAr14NGTeuIxdcEBPgyIzxD18Sxd+BATjdjG8RkerA+/4Ny5jCKzq6BHFxF3LsWCpTpnSlY8eagQ7JGL/yZSjUX0RkIFBLROoCm1X1Wf+HZkzhoKq8887P1KwZm1lBPX58J8LDQ63hnCkWfBnh7mrgP8BOnDYU54tIX1Vd6u/gjAm0DRv2ct99X/Dtt39Qr14FVq/uT3h4KGXKRAY6NGMKjC9FT+OBLqq6HkBE6uEkjrPqXMqYouD48VSeffY7xoxZSmpqBhUrRjN8+FWUKGEV1ab48SVRhJ9MEgCqukFEbNgtE7Tmz9/M/ffPZcuWgwD885+X8sIL1xIbGxXgyIwJDF8SxY8iMh3nLgKgD9YpoAlSR46coG/f2ezbd4yGDc9j2rSutGplDedM8eZLougPDAQexamjWAK86s+gjClI6ekZZGQoJUqEUqpUOBMmdCYh4TCDB7egRAnrwM8Yr4lCRBoBNYHZqjqmYEIypuCsWvUn9977OTfeWIfHH28DkDmokDHGkWPNnIj8C6f7jj7AQhH5e4FFZYyfHT6cwkMPzaN58zdYtWoX//nPGlJT0wMdljGFkrc7ij5AY1U9KiIVgbnAWwUTljH+oarMmrWehx6az65dRwgNFYYMacFTT11jxUzG5MBbokhR1aMAqrpXROy5QFOkJSWl0LPnLObN2wzAFVdUZtq0bjRten6AI/QfBMAAAB6eSURBVDOmcPOWKGp4jJUtQE3PsbNVtYdfIzMmn5UqFU5KSjplykTwwgvXcs89lxESYuNEGJMbb4ni5izTk/wZiDH+sGTJH1xwQSlq1y6PiPDWWzcQGRlGpUqlAh2aMUWGt4GLvirIQIzJT/v2HePRRxfy9turad++OgsX9kVEuPjisoEOzZgix5d2FMYXNopdoZCRocyYsZqhQxdy4MBxwsNDufrqqqSnK2FhVsxkzNnwawW1iHQWkV9FZLOIPOZlvVtEREWk6PYfZaPYBdy6dXto23YGd989hwMHjtO+fXXWrr2PJ59sS1iYPYthzNny+Y5CRCJUNSUP64cCk4EOQAKwUkTmePYb5a4Xg9Py+wdf912o2Sh2AZGYmEyLFm9y5MgJzjuvJOPGdeS22xohYncRxpyrXH9miUhzEVkLbHKnm4iIL114NMcZu2KLqp4AZgI3ZrPe08AYINn3sI1xqDqJuUyZSIYNa0X//pexceP99OnT2JKEMfnEl/vxiUA3YD+Aqv4MXOPDdpWBHR7TCe68TCLSDLhIVT/3tiMRuUdE4kUkfu/evT4c2gS7nTsPc8stH/Luu2sy540YcTVTp3ajXDnr5dWY/ORLoghR1T+yzPOlr4Psfs5llsu4DfjGAw/ntiNVfU1V41Q1rmLFij4c2gSrtLQMJkxYTt26k/n44w08+eQ3pKdnANgdhDF+4ksdxQ4RaQ6oW+/wIPCbD9slABd5TFcB/vSYjgEaAt+4H/DzgTkicoOqxvsSvCleVq7cSf/+X/Djj7sA6N69LhMndiY01CqqjfEnXxLFfTjFT1WB3cAid15uVgK1RaQ6zjCqvYDbTi5U1USgwslpEfkGeMSShMnq6NETDBu2iClTVqIKVauW4dVXr+OGG+oEOjRjioVcE4Wq7sH5ks8TVU0TkQeABUAo8JaqrhOR0UC8qs7Jc7SmWAoLC2HRoi2EhAhDhrTkySfbULKkDbJoTEHJNVGIyOt41C2cpKr35Latqs7F6XXWc94TOazbNrf9meLj998PULZsJOXLRxMREcZ//nMTkZFhNGpUKdChGVPs+FK4uwj4yv1bCpwH+Nyewpi8SElJ45lnltCw4VSGDVuUOf/yyytbkjAmQHwpevrAc1pE/gMs9FtEptj65ptt3HffF2zcuA9wnnBKT8+wympjAuxs+nqqDlyc34GY4mvPnqMMHbqQd975GYA6dcozdWpXrrmmeoAjM8aAb3UUBzlVRxECHABy7LfJmLzYt+8Y9epN5sCB40REhDJixNU8+mgrIiKsv0pjCguvn0ZxGjg0wXm8FSBDT/aZYEw+qFAhmhtvrENCwmGmTOlKrVqxgQ7JGJOF10Shqiois1X1soIKyAS3o0dPMHr0t3TtegmtWzslmFOmdCUiItRaVhtTSPlSS7hCRC71eyQm6P33v79Sv/4Uxoz5HwMGfEFGhnNzGhkZZknCmEIsxzsKEQlT1TTgKuCfIvI7cBSnDydV1eKZPGyAojzbsSORhx6az+zZGwFo1ux8pk/vZuNVG1NEeCt6WgFcCnQvoFiKBm9JwgYnOk1aWgYTJ/7AE08s5ujRVEqVCueZZ67h/vub20BCxhQh3hKFAKjq7wUUS9FiAxTl6vDhFJ5//nuOHk3l5pvr8cornalSpXSgwzLG5JG3RFFRRIbktFBVx/khHlPEHTqUTFRUGBERYcTGRjF9ejciIkLp2vWSQIdmjDlL3u7/Q4FSON2BZ/dnTCZV5b331lKnziTGjFmaOb9Hj3qWJIwp4rzdUexS1dEFFokpsn77bT8DBnzBV19tBWDJku2oqj3JZEyQyLWOwpicJCen8eKL3/Pcc99z4kQ6sbFRvPRSB+68s6klCWOCiLdE0b7AojBFzl9/HaF167fZtOkAAHfe2ZSXXupAhQrRAY7MGJPfckwUqnqgIAMpVKytRK4qVSrJRReVISwshKlTu9KmTbVAh2SM8RPreS07uSWJYtheIiNDef31VVxzTXUuuaQ8IsJ77/WgXLkowsNDAx2eMcaPLFF4Y20lAPj557/o3/8Lli9PoH376ixc2BcRoVKlUoEOzRhTACxRmBwdOXKCUaO+4ZVXlpOerlx4YQz9+8cFOixjTAGzRGGy9emnG3nwwXkkJBwmJER48MHmPPNMO0qXjgh0aMaYAmaJwpxh587D9Oo1i5SUdC677AKmTetGXNyFgQ7LGBMgligMAKmp6YSFhSAiVK5cmmefbUd4eCgDBlxuY1YbU8zZN4Dhf//bwWWXvca7767JnPfww1fy4INXWJIwxliiKM4OHDjOvff+l1at3mLt2j1MmRKPjXRrjMkqeIqerJGcz1SVd99dw8MPf8nevccoUSKERx9txYgRV1vXG8aYMwRPosjvJBGkjep27z5C794fs3jxNgDatLmYqVO7Uq9excAGZowptIInUZxkjeS8Kls2kl27jlChQjRjx3agX78mdhdhjPEq+BKFOcPChb9z6aUXUL58NBERYXz00a1ccEEpype3DvyMMbmzyuwgtmtXEr17f0zHju8ybNiizPkNG55nScIY4zO7owhC6ekZTJ++iuHDv+Lw4RSiosKoU6e8DSZkjDkrliiCzI8/7qJ//89ZufJPALp2rc2kSV2oVq1sgCMzxhRVliiCyLZth2je/HXS05XKlWOYOPE6brqprt1FGGPOiV8ThYh0BiYAocAbqvpCluVDgH8AacBe4O+q+oc/Ywpm1aqV5a67mhITE8FTT7UlJsY68DPGnDu/VWaLSCgwGbgOqA/0FpH6WVb7CYhT1cbALGCMv+IJRtu2HeL669/n22+3Zc577bXrGTeukyUJY0y+8ecdRXNgs6puARCRmcCNwPqTK6jqYo/1lwO3+zGeoJGams64cct46qlvOX48jX37jrFs2d0AVsxkjMl3/kwUlYEdHtMJwBVe1r8bmJfdAhG5B7gHoGrVqvkVX5H0/ffb6d//c9at2wtAr14NGTeuY4CjMsYEM38miux+2mbbbFpEbgfigDbZLVfV14DXAOLi4opl0+uDB48zdOhC3nzzJwBq1izHlCld6dixZoAjM8YEO38migTgIo/pKsCfWVcSkWuBEUAbVU3xYzxFWkaG8tlnv1KiRAiPPXYVw4dfRVRUiUCHZYwpBvyZKFYCtUWkOrAT6AXc5rmCiDQDpgOdVXWPH2MpkjZu3Ef16mWJiAijfPlo/u//elC1ahnq1q0Q6NCMMcWI3556UtU04AFgAbAB+FBV14nIaBG5wV3tJaAU8JGIrBaROf6Kpyg5diyVESO+onHjqYwZszRzfseONS1JGGMKnF/bUajqXGBulnlPeLy+1p/HL4rmz9/MgAFfsHXrIQD27TsW4IiMMcWdtcwuJP78M4lBg+bz0UfO08ONGp3HtGnduPLKi3LZ0hhj/MsSRSHw22/7iYt7jaSkE0RHl2DUqDYMGtSCEiVCAx2aMcZYoigMateO5fLLK1OyZAleffU6Lr7YOvAzxhQeligC4PDhFJ54YjEDBlzOJZeUR0SYM6cXJUuGBzo0Y4w5gyWKAqSqzJq1nocems+uXUfYuHEf8+c7vZZYkjDGFFaWKArIli0HeeCBucybtxmAFi2q8OKL9tCXMabws0ThZydOpDN27P94+uklJCenUbZsJC+80J5//vMyQkKsAz9jTOFnicLPduxIZPTob0lJSadPn0a8/HJHKlUqFeiwjDHGZ5Yo/ODgweOULRuJiFCzZiwTJnSmVq1Y2revEejQjDEmz4peoti9Cl4unEU2GRnKjBmrGTp0Ia+80om+fZsAcO+9cQGOzBhjzp7f+noKiOpdAnbodev20LbtDO6+ew4HDhzPrLQ2xpiirujdUQA8XHiGpDh2LJWnn/6WsWOXkZaWwXnnlWT8+E707t0w0KEZY0y+KJqJopD47bf9dOr0Ltu2HUIE+ve/jOeea0+5clGBDs0YY/KNJYpzcPHFZYiMDKNJk0pMm9aNFi2qBDokU4ikpqaSkJBAcnJyoEMxxUhkZCRVqlShRIn8G9jMEkUepKVlMG1aPL17N6R8+WgiIsKYP78PlSuXJiwsuKp7zLlLSEggJiaGatWqIVI4H8AwwUVV2b9/PwkJCVSvXj3f9mvfbj5asWInzZu/zoMPzmPYsEWZ8y++uKwlCZOt5ORkypcvb0nCFBgRoXz58vl+F2t3FLlITExmxIivmTJlJapQtWoZbryxTqDDMkWEJQlT0PzxnrNEkQNV5YMP1jF48AL++usIYWEhDBnSgieeaGMd+BljihUrM8nBzz/vpnfvj/nrryNceeVF/PjjPbz4YgdLEqZICQ0NpWnTpjRs2JDrr7+eQ4cOZS5bt24d7dq145JLLqF27do8/fTTqJ569HzevHnExcVRr1496tatyyOPPBKIU/Dqp59+4h//+Eegw/Dq+eefp1atWtSpU4cFCxZku87VV19N06ZNadq0KRdeeCHdu3cH4ODBg9x00000btyY5s2b88svvwBw4sQJWrduTVpaWsGchKoWqb/LqqD+kpaWftr04MHz9fXXV2l6eobfjmmC1/r16wMdgpYsWTLzdb9+/fSZZ55RVdVjx45pjRo1dMGCBaqqevToUe3cubNOmjRJVVXXrl2rNWrU0A0bNqiqampqqk6ePDlfY0tNTT3nfdxyyy26evXqAj1mXqxbt04bN26sycnJumXLFq1Ro4ampaV53aZHjx7673//W1VVH3nkER01apSqqm7YsEHbtWuXud6oUaP03XffzXYf2b33gHg9y+9dK3pyLV68lQED5jJ9ejdat74YgHHjOgU4KhM0/NXtTB4an7Zs2ZI1a9YA8N5779GqVSs6duwIQHR0NJMmTaJt27bcf//9jBkzhhEjRlC3bl0AwsLCGDBgwBn7PHLkCA8++CDx8fGICE8++SQ333wzpUqV4siRIwDMmjWLzz//nBkzZnDnnXcSGxvLTz/9RNOmTZk9ezarV6+mbFlnVMdatWqxdOlSQkJC6N+/P9u3bwfglVdeoVWrVqcdOykpiTVr1tCkidNVzooVKxg0aBDHjx8nKiqKt99+mzp16jBjxgy++OILkpOTOXr0KF9//TUvvfQSH374ISkpKdx000089dRTAHTv3p0dO3aQnJzMQw89xD333OPz9c3OZ599Rq9evYiIiKB69erUqlWLFStW0LJly2zXT0pK4uuvv+btt98GYP369QwfPhyAunXrsm3bNnbv3k2lSpXo3r07w4cPp0+fPucUoy+KfaLYs+coQ4cu5J13fgZg3LhlmYnCmGCRnp7OV199xd133w04xU6XXXbZaevUrFmTI0eOcPjwYX755RcefvjhXPf79NNPU6ZMGdauXQs4RSW5+e2331i0aBGhoaFkZGQwe/Zs7rrrLn744QeqVatGpUqVuO222xg8eDBXXXUV27dvp1OnTmzYsOG0/cTHx9Ow4akeEOrWrcuSJUsICwtj0aJF/Otf/+Ljjz8GYNmyZaxZs4bY2Fi+/PJLNm3axIoVK1BVbrjhBpYsWULr1q156623iI2N5fjx41x++eXcfPPNlC9f/rTjDh48mMWLF59xXr169eKxxx47bd7OnTtp0aJF5nSVKlXYuXNnjtdm9uzZtG/fntKlSwPQpEkTPvnkE6666ipWrFjBH3/8QUJCApUqVaJhw4asXLky1+udH4ptosjIUN5880eGDVvEwYPJRESEMnJka4YOvTLQoZlgFKBuZ44fP07Tpk3Ztm0bl112GR06dACcIuecno7Jy1MzixYtYubMmZnT5cqVy3WbW2+9ldDQUAB69uzJ6NGjueuuu5g5cyY9e/bM3O/69esztzl8+DBJSUnExMRkztu1axcVK1bMnE5MTOSOO+5g06ZNiAipqamZyzp06EBsbCwAX375JV9++SXNmjUDnLuiTZs20bp1ayZOnMjs2bMB2LFjB5s2bTojUYwfP963iwOn1fmc5O36vv/++6fVuTz22GM89NBDNG3alEaNGtGsWTPCwpyv7dDQUMLDw8+4Lv5QLBPF1q0Huf322fzvfzsA6NixJpMnd6FWrdgAR2ZM/oqKimL16tUkJibSrVs3Jk+ezMCBA2nQoAFLliw5bd0tW7ZQqlQpYmJiaNCgAatWrcos1slJTgnHc17WZ/pLliyZ+bply5Zs3ryZvXv38umnnzJy5EgAMjIyWLZsGVFROXeHExUVddq+H3/8ca655hpmz57Ntm3baNu2bbbHVFWGDx/Ovffee9r+vvnmGxYtWsSyZcuIjo6mbdu22bZHyMsdRZUqVdixY0fmdEJCAhdeeGG257N//35WrFiRmagASpcunVkMpapUr179tIZ0KSkpREZGZru//FQsn3oqXTqC337bz/nnl2LmzJuZP7+PJQkT1MqUKcPEiRMZO3Ysqamp9OnTh++//55Fi5zGo8ePH2fgwIE8+uijAAwdOpTnnnuO3377DXC+uMeNG3fGfjt27MikSZMyp08WPVWqVIkNGzZkFi3lRES46aabGDJkCPXq1cv89Z51v6tXrz5j23r16rF586lemhMTE6lcuTIAM2bMyPGYnTp14q233sqsQ9m5cyd79uwhMTGRcuXKER0dzcaNG1m+fHm2248fP57Vq1ef8Zc1SQDccMMNzJw5k5SUFLZu3cqmTZto3rx5tvv96KOP6Nat22lf/IcOHeLEiRMAvPHGG7Ru3TqzWGr//v1UrFgxX7vqyEmxSRQLFmwmJcV5lKx8+WjmzOnFxo3307NnQ2sUZYqFZs2a0aRJE2bOnElUVBSfffYZzzzzDHXq1KFRo0ZcfvnlPPDAAwA0btyYV155hd69e1OvXj0aNmzIrl27ztjnyJEjOXjwIA0bNqRJkyaZv7RfeOEFunXrRrt27bjgggu8xtWzZ0/efffdzGIngIkTJxIfH0/jxo2pX78+06ZNO2O7unXrkpiYSFJSEgCPPvoow4cPp1WrVqSnp+d4vI4dO3LbbbfRsmVLGjVqxC233EJSUhKdO3cmLS2Nxo0b8/jjj59Wt3C2GjRowN/+9jfq169P586dmTx5cmaxW5cuXfjzzz8z1505cya9e/c+bfsNGzbQoEED6taty7x585gwYULmssWLF9OlS8EMrSDZlaEVZnEXicbv8D3mHTsSGThwPp9+upGnn76GkSNb+zE6Y07ZsGED9erVC3QYQW38+PHExMQU+rYU/tCjRw+ef/556tQ5s6eI7N57IrJKVc9qFLWgvaNIS8tg3Lhl1Ks3mU8/3UipUuHExlr338YEk/vuu4+IiIhAh1HgTpw4Qffu3bNNEv4QlJXZy5cn0L//5/z8824Abr65HhMmdKZy5dIBjswYk58iIyPp27dvoMMocOHh4fTr16/Ajhd0ieKHHxK48so3UYVq1coyadJ1dO16SaDDMsWUt8dQjfEHf1QnBF2iaN68Mp061aJZs/MZObI10dH+fyLAmOxERkayf/9+62rcFBh1x6PI70dmi3xl9qZN+xk8eAHjxnXikkucR+syMpSQEPtgmsCyEe5MIOQ0wt25VGYX2TuKlJQ0Xnjhe55//ntSUtKJjAxj1qy/AViSMIVCiRIl8nWUMWMCxa9PPYlIZxH5VUQ2i8gZrVFEJEJEPnCX/yAi1XzZ71dfbaFx42mMGvUtKSnp3HVXU6ZN65bf4RtjjMGPdxQiEgpMBjoACcBKEZmjqus9VrsbOKiqtUSkF/Ai0PPMvZ2y9UBZrr32PwDUq1eBadO6WSd+xhjjR/68o2gObFbVLap6ApgJ3JhlnRuBf7uvZwHtJZdav4PHooiMDOO559qxenV/SxLGGONnfqvMFpFbgM6q+g93ui9whao+4LHOL+46Ce707+46+7Ls6x7gZMfwDYFf/BJ00VMB2JfrWsWDXYtT7FqcYtfilDqqelbdzPqzMju7O4OsWcmXdVDV14DXAEQk/mxr7oONXYtT7FqcYtfiFLsWp4hI/Nlu68+ipwTgIo/pKsCfOa0jImFAGeCAH2MyxhiTR/5MFCuB2iJSXUTCgV7AnCzrzAHucF/fAnytRa1hhzHGBDm/FT2papqIPAAsAEKBt1R1nYiMxhnkew7wJvAfEdmMcyfRy4ddv+avmIsguxan2LU4xa7FKXYtTjnra1HkWmYbY4wpWEHbzbgxxpj8YYnCGGOMV4U2Ufir+4+iyIdrMURE1ovIGhH5SkSCthVibtfCY71bRERFJGgfjfTlWojI39z3xjoRea+gYywoPnxGqorIYhH5yf2cFMwYogVMRN4SkT1uG7XslouITHSv0xoRudSnHatqofvDqfz+HagBhAM/A/WzrDMAmOa+7gV8EOi4A3gtrgGi3df3Fedr4a4XAywBlgNxgY47gO+L2sBPQDl3+rxAxx3Aa/EacJ/7uj6wLdBx++latAYuBX7JYXkXYB5OG7YWwA++7Lew3lH4pfuPIirXa6Gqi1X1mDu5HKfNSjDy5X0B8DQwBgjm/r19uRb/BCar6kEAVd1TwDEWFF+uhQInh7gsw5ltuoKCqi7Be1u0G4F31LEcKCsiF+S238KaKCoDOzymE9x52a6jqmlAIlC+QKIrWL5cC0934/xiCEa5XgsRaQZcpKqfF2RgAeDL++IS4BIRWSoiy0Wkc4FFV7B8uRajgNtFJAGYCzxYMKEVOnn9PgEK73gU+db9RxDw+TxF5HYgDmjj14gCx+u1EJEQYDxwZ0EFFEC+vC/CcIqf2uLcZX4nIg1V9ZCfYytovlyL3sAMVX1ZRFritN9qqKoZ/g+vUDmr783Cekdh3X+c4su1QESuBUYAN6hqSgHFVtByuxYxOJ1GfiMi23DKYOcEaYW2r5+Rz1Q1VVW3Ar/iJI5g48u1uBv4EEBVlwGROB0GFjc+fZ9kVVgThXX/cUqu18ItbpmOkySCtRwacrkWqpqoqhVUtZqqVsOpr7lBVc+6M7RCzJfPyKc4DzogIhVwiqK2FGiUBcOXa7EdaA8gIvVwEsXeAo2ycJgD9HOffmoBJKrqrtw2KpRFT+q/7j+KHB+vxUtAKeAjtz5/u6reELCg/cTHa1Es+HgtFgAdRWQ9kA4MVdX9gYvaP3y8Fg8Dr4vIYJyiljuD8YeliLyPU9RYwa2PeRIoAaCq03DqZ7oAm4FjwF0+7TcIr5Uxxph8VFiLnowxxhQSliiMMcZ4ZYnCGGOMV5YojDHGeGWJwhhjjFeWKEyhIyLpIrLa46+al3Wr5dRTZh6P+Y3b++jPbpcXdc5iH/1FpJ/7+k4RudBj2RsiUj+f41wpIk192GaQiESf67FN8WWJwhRGx1W1qcfftgI6bh9VbYLT2eRLed1YVaep6jvu5J3AhR7L/qGq6/MlylNxTsG3OAcBlijMWbNEYYoE987hOxH50f27Mpt1GojICvcuZI2I1Hbn3+4xf7qIhOZyuCVALXfb9u4YBmvdvv4j3PkvyKkxQMa680aJyCMicgtOn1v/5x4zyr0TiBOR+0RkjEfMd4rIq2cZ5zI8OnQTkakiEi/O2BNPufMG4iSsxSKy2J3XUUSWudfxIxEplctxTDFnicIURlEexU6z3Xl7gA6qeinQE5iYzXb9gQmq2hTnizrB7a6hJ9DKnZ8O9Mnl+NcDa0UkEpgB9FTVRjg9GdwnIrHATUADVW0MPOO5sarOAuJxfvk3VdXjHotnAT08pnsCH5xlnJ1xuuk4aYSqxgGNgTYi0lhVJ+L05XONql7jduUxErjWvZbxwJBcjmOKuULZhYcp9o67X5aeSgCT3DL5dJx+i7JaBowQkSrAJ6q6SUTaA5cBK93uTaJwkk52/k9EjgPbcLqhrgNsVdXf3OX/Bu4HJuGMdfGGiHwB+NyluaruFZEtbj87m9xjLHX3m5c4S+J0V+E5QtnfROQenM/1BTgD9KzJsm0Ld/5S9zjhONfNmBxZojBFxWBgN9AE5074jEGJVPU9EfkB6AosEJF/4HSr/G9VHe7DMfp4diAoItmOb+L2LdQcp5O5XsADQLs8nMsHwN+AjcBsVVVxvrV9jhNnFLcXgMlADxGpDjwCXK6qB0VkBk7Hd1kJsFBVe+chXlPMWdGTKSrKALvc8QP64vyaPo2I1AC2uMUtc3CKYL4CbhGR89x1YsX3McU3AtVEpJY73Rf41i3TL6Oqc3EqirN78igJp9vz7HwCdMcZI+EDd16e4lTVVJwipBZusVVp4CiQKCKVgOtyiGU50OrkOYlItIhkd3dmTCZLFKaomALcISLLcYqdjmazTk/gFxFZDdTFGfJxPc4X6pcisgZYiFMskytVTcbpXfMjEVkLZADTcL50P3f39y3O3U5WM4BpJyuzs+z3ILAeuFhVV7jz8hynW/fxMvCIqv6MMz72OuAtnOKsk14D5onIYlXdi/NE1vvucZbjXCtjcmS9xxpjjPHK7iiMMcZ4ZYnCGGOMV5YojDHGeGWJwhhjjFeWKIwxxnhlicIYY4xXliiMMcZ49f+t84JzzoTh4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# Import some data to play with\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data\n",
    "y = iris.target\n",
    "\n",
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[0, 1, 2])\n",
    "n_classes = y.shape[1]\n",
    "\n",
    "# Add noisy features to make the problem harder\n",
    "random_state = np.random.RandomState(0)\n",
    "n_samples, n_features = X.shape\n",
    "X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]\n",
    "\n",
    "# shuffle and split training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5,random_state=0)\n",
    "\n",
    "# Learn to predict each class against the other\n",
    "classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,\n",
    "                                 random_state=random_state))\n",
    "y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
    "\n",
    "# Compute the prob\n",
    "y_prob = classifier.predict_proba(X_test)\n",
    "#print(y_prob)\n",
    "\n",
    "# Compute ROC curve and ROC area for each class\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "# Compute micro-average ROC curve and ROC area\n",
    "fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n",
    "roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr[2], tpr[2], color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 几个回归的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9571734475374732"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import explained_variance_score\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "explained_variance_score(y_true, y_pred)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "mean_absolute_error(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.375"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "mean_squared_error(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9486081370449679"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " from sklearn.metrics import r2_score\n",
    " y_true = [3, -0.5, 2, 7]\n",
    " y_pred = [2.5, 0.0, 2, 8]\n",
    " r2_score(y_true, y_pred)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
